# -*- coding: utf-8 -*-
'''
Created on 2011-11-21

@author: Administrator
'''
import struct
import os
from setting import *
import time
import datetime
import urllib2
import simplejson as json
def myfloat(src):
    if len(src.strip())==0:return 0
    return float(src)
def str2date(src):
#    print src
    return datetime.datetime(int(src[:4]),int(src[4:6]),int(src[6:8]),9,0,0)

def getstockhistory(stockcode,reg):
    if reg in ['SH','SZ']:reg='CH'
    if reg in ['NS','NQ','AS']:reg='US'
    stockcode=stockcode+':'+reg
    if reg=='HK':stockcode=stockcode.strip('0')
#        print 'thread stockcode:',stockcode
    url='http://www.bloomberg.com/apps/data?pid=webpxta&Securities='+stockcode+'&TimePeriod=5Y&Outfields=HDATE,PR006-H,PR007-H,PR008-H,PR005-H,PR013-H'
#    print url
    content=''
    try:
        content=urllib2.urlopen(url).read()
    except:
        print '[x]',url   
    if content[:5]=='Error':
        print '[r]',#url

    else:
        clist=content.split('\n')
        tlist=[]
        for t in range(1,len(clist)-2):
            r={}
            klist=clist[t].split('"')
            r['date']=int(klist[0])
            r['open']=myfloat(klist[1])
            r['high']=myfloat(klist[2])
            r['low']=myfloat(klist[3])
            r['close']=myfloat(klist[4])
            r['vol']=myfloat(klist[5])
            tlist.append(r)
        return tlist
class dayProcess():
    yearnumber=250
    def __init__(self):
        self.datalist=[]
        self.dresult=[]
        self.closelist=[]
        self.highlist=[]
        self.lowlist=[]
        self.datalen=0    
#        print self.dresult
    def isnew(self,newdata):
        self.datalen=len(self.datalist)
        if self.datalen==0:return True
        if self.datalist[self.datalen-1]['date']==newdata['date']:return False
        return True
    def removeone(self):
        olddata=self.datalist.pop()
        self.closelist.pop()
        self.highlist.pop()
        self.lowlist.pop()
        self.dresult.pop()
        return olddata
    def getma(self,mv):
        self.datalen=len(self.datalist)
        if self.datalen<mv:return 0.0
        return sum(self.closelist[self.datalen-mv:self.datalen])/mv
    def getxgxd(self):
        self.datalen=len(self.datalist)
        if self.datalen<self.yearnumber:return (0.0,0.0)
        return (max(self.highlist[self.datalen-self.yearnumber:self.datalen]),min(self.lowlist[self.datalen-self.yearnumber:self.datalen]))
    def add(self,newdata):
        if self.isnew(newdata)==False:
            olddata=self.removeone()
#            print 'pop date',olddata['date']
            newdata['open']=olddata['open']
            newdata['high']=max(olddata['high'],newdata['high'])
            newdata['low']=min(olddata['low'],newdata['low'])
            self.datalist.append(newdata)
            self.closelist.append(newdata['close'])
            self.highlist.append(newdata['high'])
            self.lowlist.append(newdata['low'])
            return self.calc(newdata)              
        else:
            self.datalist.append(newdata)
            self.closelist.append(newdata['close'])
            self.highlist.append(newdata['high'])
            self.lowlist.append(newdata['low'])
            return self.calc(newdata)  
    def calc(self,newdata):
        r={}
        r['ma20']=self.getma(20)
        r['ma60']=self.getma(60)
        r['ma120']=self.getma(120)
        (r['xg'],r['xd'])=self.getxgxd()
        r['date']=newdata['date']
        r['close']=newdata['close'] 
        self.dresult.append(r)
        return r
    def getresult(self):
        return self.dresult
    pass
class monthProcess(dayProcess):
    yearnum=12
    def isnew(self,newdata):
        self.datalen=len(self.datalist)
        if self.datalen==0:return True
#        print self.weeklist
        if self.datalist[self.datalen-1]['date']/100==newdata['date']/100:return False
        return True          
    pass
    def calc(self,newdata):
        r={}
        r['ma20']=self.getma(20)
        r['ma60']=self.getma(60)
        r['ma120']=self.getma(120)
        (r['xg'],r['xd'])=self.getxgxd()
        r['date']=newdata['date']/100
        r['close']=newdata['close'] 
        self.dresult.append(r)
        return r        
    pass
def getweeklist():
    startdate=datetime.date(2020,1,1)
    enddate=datetime.date(2000,1,1)
    daylist={}
    weekindex=0
    while startdate>enddate:
#            print startdate
        daylist[startdate.year*10000+startdate.month*100+startdate.day]=weekindex       
        newdate=startdate-datetime.timedelta(days=1)        
        startdate=newdate
        if startdate.weekday()==4:weekindex=startdate.year*10000+startdate.month*100+startdate.day
    return daylist
class weekProcess(dayProcess):
    yearnumber=54
    weeklist=getweeklist()
    def isnew(self,newdata):
        self.datalen=len(self.datalist)
        if self.datalen==0:return True
#        print self.weeklist
        if self.weeklist[self.datalist[self.datalen-1]['date']]==self.weeklist[newdata['date']]:return False
        return True          
    pass
    def calc(self,newdata):
        r={}
        r['ma20']=self.getma(20)
        r['ma60']=self.getma(60)
        r['ma120']=self.getma(120)
        (r['xg'],r['xd'])=self.getxgxd()
        r['date']=self.weeklist[newdata['date']]
        r['close']=newdata['close'] 
        self.dresult.append(r)
        return r
class processByDate():
    def __init__(self):
#        self.mp=monthProcess()
#        self.wp=weekProcess()
#        self.dp=dayProcess()
        if os.path.exists(stockprocessbydatepath)==False: os.mkdir(stockprocessbydatepath)
        for reg in reglist:
            regpath=os.path.join(stockprocessbydatepath,reg)
            if os.path.exists(regpath)==False: os.mkdir(regpath)
            for d in ['day','week','month']:
                dpath=os.path.join(stockprocessbydatepath,reg,d)
                if os.path.exists(dpath)==False: os.mkdir(dpath)
        
        pass
    def getfilelist(self):
        stocklist={}
        for root,dirs,files in os.walk(rawcontentpath):
            for f in files:
#                print f
#                print os.path.join(root,f)
                if f.find('.'): stockcode=f[:-4]
                else: stockcode=f
                stocklist[stockcode]=os.path.join(root,f)
#        print stocklist
        return stocklist
    def processonestock(self,stockcode,stockfilename):
        def loadstockfromfile(stockfilename):
            clist=open(stockfilename,'r').read().split('\n')
            tlist=[]
            for t in range(1,len(clist)-2):
                r={}
                klist=clist[t].split('"')
                r['date']=int(klist[0])
                r['open']=myfloat(klist[1])
                r['high']=myfloat(klist[2])
                r['low']=myfloat(klist[3])
                r['close']=myfloat(klist[4])
                r['vol']=myfloat(klist[5])
                tlist.append(r)
            return tlist        
        stockdata=loadstockfromfile(stockfilename)
        for daydata in stockdata:
            self.dp.add(daydata)
            self.wp.add(daydata)
            self.mp.add(daydata)
        dret=self.dp.getresult()
        wret=self.wp.getresult()
        mret=self.mp.getresult()        
        
        for d in dret:
            reg=stockcode[:2]
#            print reg,d
            fname=os.path.join(stockprocessbydatepath,reg,'day',str(d['date']))
            if os.path.exists(fname)==False: 
                f=open(fname,'w')
                f.close()
            f=open(fname,'a')
            d['code']=stockcode
            f.write(json.dumps(d))
            f.write('\n')
            f.close
        for d in wret:
            reg=stockcode[:2]
#            print reg,d
            fname=os.path.join(stockprocessbydatepath,reg,'week',str(d['date']))
            if os.path.exists(fname)==False: 
                f=open(fname,'w')
                f.close()
            f=open(fname,'a')
            d['code']=stockcode
            f.write(json.dumps(d))
            f.write('\n')
            f.close        
        for d in mret:
            reg=stockcode[:2]
#            print reg,d
            fname=os.path.join(stockprocessbydatepath,reg,'month',str(d['date']/100))
            if os.path.exists(fname)==False: 
                f=open(fname,'w')
                f.close()
            f=open(fname,'a')
            d['code']=stockcode
            f.write(json.dumps(d))
            f.write('\n')
            f.close                        
        
        
        print stockcode,len(stockdata),len(dret),len(wret),len(mret)
        pass
    def process(self,reg):
        def loadstockfromfile(stockfilename):
            clist=open(stockfilename,'r').read().split('\n')
            tlist=[]
            for t in range(0,len(clist)-1):
                r={}
                klist=clist[t].split('"')
                r['date']=int(klist[0])
                r['open']=myfloat(klist[1])
                r['high']=myfloat(klist[2])
                r['low']=myfloat(klist[3])
                r['close']=myfloat(klist[4])
                r['vol']=myfloat(klist[5])
                tlist.append(r)
            return tlist   
        stocklist=self.getfilelist()
        daylist={}
        weeklist={}
        monthlist={}
        i=0
        for code in sorted(stocklist.keys()):
            dp=dayProcess()
            wp=weekProcess()
            mp=monthProcess()
            i=i+1
#            print code,stocklist[code],i,starttime,(datetime.datetime.now()-starttime).seconds
            datalist=loadstockfromfile(stocklist[code])
            while len(datalist)>0:
                oneday=datalist.pop(0)
                dp.add(oneday)
                wp.add(oneday)
                mp.add(oneday) 
            dret=dp.getresult()
            dp=None
            wret=wp.getresult()
            wp=None
            mret=mp.getresult()  
            mp=None
            for d in dret:
                d['code']=code
                if d['date'] in daylist.keys():
                    daylist[d['date']].append(d)
                else:
                    daylist[d['date']]=[]
                    daylist[d['date']].append(d)   
            dret=None
            for d in wret:
                d['code']=code
                if d['date'] in weeklist.keys():
                    weeklist[d['date']].append(d)
                else:
                    weeklist[d['date']]=[]
                    weeklist[d['date']].append(d) 
            wret=None 
            for d in mret:
                d['code']=code
                if d['date'] in monthlist.keys():
                    monthlist[d['date']].append(d)
                else:
                    monthlist[d['date']]=[]
                    monthlist[d['date']].append(d) 
            mret=None
        print starttime,(datetime.datetime.now()-starttime).seconds
######################################################################  
        self.removelastfile(reg,'day')
        self.removelastfile(reg,'week')
        self.removelastfile(reg,'month')

        self.dump2file(reg, 'day', daylist)
        self.dump2file(reg, 'week', weeklist)
        self.dump2file(reg, 'month', monthlist)
    def removelastfile(self,reg,duration):        
        dlist=[]
        dpath=os.path.join(stockprocessbydatepath,reg,duration)
#        print dpath
        for root,dirs,files in os.walk(dpath):
            for f in files:
                dlist.append({'code':f,'filename':os.path.join(root,f)})
        lastone=dlist.pop()
#        print 'last ',duration,lastone['filename']     
        os.remove(lastone['filename'])
    def dump2file(self,reg,duration,datalist):
        for d in datalist.keys():
            fname=os.path.join(stockprocessbydatepath,reg,duration,str(d))
            if os.path.exists(fname)==False:
                f=open(fname,'w')            
                f.write(json.dumps(datalist[d]))
                f.close           
        pass
def downloadandprocessonestock():
    stockcode='000001'
    reg='SZ'
    weeklist=getweeklist()
    ret=getstockhistory(stockcode,reg)
    dp=monthProcess()
#    dp=weekProcess()
#    dp=dayProcess()
    i=0
    for rr in ret:
#        print rr['date']
        rett=dp.add(rr)        
#        print rett['date']
#        print '--------------------------'
#        print dp.getresult()
        i=i+1
#        if i==200: break
        
    dret=dp.getresult()
    for rr in dret:
        print rr   
class outputResult():
    def getdatelist(self,reg,duration):
        dlist=[]
        dpath=os.path.join(stockprocessbydatepath,reg,duration)
        print dpath
        for root,dirs,files in os.walk(dpath):
            for f in files:
                dlist.append({'code':f,'filename':os.path.join(root,f)})
        return dlist
    def parseonedayma(self,filename):
        clist=json.load(open(filename,'r'))
        
        result={}
        duralist=['ma120','ma60','ma20']
        for dura in duralist:
            result['close>'+dura]=[]
        result['count']=len(clist)        
        for e in clist:
            r={}
            
            for mv in ['ma120','ma60','ma20']:
                if e[mv]>0:
                    if e['close']>e[mv] :result['close>'+mv].append(e['code'])

            
        return result
    def outputM120(self,reg):
        f=open('c:\\output.xls','w')
        dlist=self.getdatelist(reg, 'week')
        for d in dlist:
            result=self.parseonedayma(d['filename'])
            print d['code'],result['count'],len(result['close>ma120']),len(result['close>ma60']),len(result['close>ma20'])
            f.write(d['code'])
            f.write('\t')
            f.write(str(result['count']))
            f.write('\t')
            f.write(str(len(result['close>ma120'])))
            f.write('\t')
            f.write(str(len(result['close>ma60'])))
            f.write('\t')
            f.write(str(len(result['close>ma20'])))
            f.write('\n')
        f.close()
        
#        print regpath
if __name__ == '__main__':
    starttime=datetime.datetime.now()
    print starttime   
#    downloadandprocessonestock()
#    pbd=processByDate()
#    pbd.process('HK')
    out=outputResult()
    out.outputM120('SH')
    print 'download OK',starttime,(datetime.datetime.now()-starttime).seconds
    pass